package common.main.cnkiOverseasPatent;

import common.http.HtmlInfo;
import common.http.SimpleHttp;
import common.main.cnkiPatent.*;
import common.main.google.NotFoundException;
import common.system.FileOperation;
import common.utils.DomTree;
import org.apache.log4j.Logger;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.NodeList;

import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;

public class CnkiOverseasPatentMain {


    static Logger logger = Logger.getLogger(CnkiPatentMain.class);

    static String cnkiUrlPrefix = "http://nvsm.cnki.net";
    static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36";
    //采集标题包含某关键词的url
    static String modelUrl = "http://nvsm.cnki.net/KNS/request/SearchHandler.ashx";
    static String[] paramsModel = {"action##","NaviCode##*","catalogName##SOPD_IPCCLS","ua##1.21","isinEn##0","PageName##ASP.brief_result_aspx",
            "DbPrefix##SOPD","DbCatalog##国外专利数据库","ConfigFile##SOPD.xml","db_opt##SCOD","db_value##国外专利数据库",
            "date_gkr_from##2010-01-01","date_gkr_to##2018-12-31","txt_2_sel##TI","txt_2_value1##<searchKeyword>","txt_2_logical##and",
            "txt_2_relation###CNKI_AND","txt_2_special1##=","his##0"};
    static String allPublicNum;
    static HtmlInfo html = new HtmlInfo();
    static SimpleHttp http = new SimpleHttp();
    static String searchKeyword;


    //searchKeyword文件每行的格式： 关键词##1986-01-01,2011-12-31;2012-01-01,2013-12-31。确保每个时间段内的数据都在6000条之内。



    public static void main(String[] args) throws InterruptedException {

        allPublicNum = CnkiOverseasPatentSaveDataToSql.getAllPublicNum();

        html.setUa(ua);
        html.setEncode("utf-8");

        CnkiPatentConfig.init();
        String url = "";
        if(CnkiPatentConfig.entranceUrl != null){
            searchKeyword = CnkiPatentConfig.searchKeyword;
            html.setOrignUrl(modelUrl);
            //将params中的searchkeyword填充进去
            String [] params = paramsModel.clone();
            params[14] = paramsModel[14].replace("<searchKeyword>",searchKeyword);
            http.simplePost(html, new ArrayList<>(Arrays.asList(params)));
            url = CnkiPatentConfig.entranceUrl;
            queryList(url);
            //执行完的关键词存储下来
            FileOperation.appendWrite(searchKeyword + ";","config/CnkiOverseasPatentExecutedKeywords.txt");
            searchKeyword = "";
        }

        while(true){
            //通过数据库读取关键词
            List<String> lines = CnkiOverseasPatentSaveDataToSql.getAllSearchKeywords();//FileOperation.read("config/cnkiPatentSearchKeyword.txt").split("\n");//{"人工智能##1986-01-01,2011-12-31;2012-01-01,2013-12-31"};
            //读取出已经执行完的关键词
            String executedKeywords = FileOperation.read("config/CnkiOverseasPatentExecutedKeywords.txt");
            if(executedKeywords == null){
                executedKeywords = "";
            }
            //标题关键字查询专利
            for (int i = 0; i < lines.size(); i++) {
                if(executedKeywords.contains(lines.get(i))){
                    continue;
                }
                searchKeyword = lines.get(i);
                //将params中的searchkeyword填充进去
                String [] params = paramsModel.clone();
                params[14] = paramsModel[14].replace("<searchKeyword>",lines.get(i));
                html.setOrignUrl(modelUrl);
                http.simplePost(html,new ArrayList<>(Arrays.asList(params)));
                //获取列表信息的url
                url = "http://nvsm.cnki.net/kns/brief/brief.aspx?keyValue=&S=1&sorttype=&pagename=" + html.getContent();
                queryList(url);
                //执行完的关键词存储下来
                FileOperation.appendWrite(searchKeyword + ";","config/CnkiOverseasPatentExecutedKeywords.txt");
            }

            //获取每个关键词的最新的海外专利数据：按公开日排序，翻页到该页数据都在数据库中则跳过该关键词。
            for(int i = 0;i < lines.size();i++){
                searchKeyword = lines.get(i);
                logger.info("获取关键词《" + searchKeyword + "》的最新海外专利信息");
                //将params中的searchkeyword填充进去
                String [] params = paramsModel.clone();
                params[14] = paramsModel[14].replace("<searchKeyword>",lines.get(i));
                html.setOrignUrl(modelUrl);
                http.simplePost(html,new ArrayList<>(Arrays.asList(params)));
                String newUrl = "http://nvsm.cnki.net/kns/brief/brief.aspx?keyValue=&S=1&pagename=" + html.getContent() + "&sorttype=(%e5%85%ac%e5%bc%80%e6%97%a5%2c+%27DATE%27)+desc";
                queryNewPatent(newUrl);
            }
            //关键词全部查询完之后，等待一天
            logger.info("关键词全部采集完毕，等待24小时后继续采集,下次开始时间为："+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(new Date().getTime() + 1000 * 60 * 60 * 24)));
            Thread.sleep(1000 * 60 * 60 * 24);

        }
    }

    public static void queryList(String url) throws InterruptedException {

        //重新启动次数   判断依据为：如果下一页的url获取失败，那么就重新获取该页的数据，重新获取一次
        int restartNum = 0;

        restart:while (url != null || restartNum <= 1){
            //记录下当前url，当程序出现问题时重新请求该url
            String restartUrl = url;
            CnkiPatentConfig.entranceUrl = url;
            logger.info("当前列表URL为：" + url);
            FileOperation.appendWrite("列表url为：" + url + "\n","config/cnkiPatentUrl.txt");
            logger.info("当前关键词为：" + searchKeyword);
            FileOperation.appendWrite("当前关键词为：" + searchKeyword + "\n","config/cnkiPatentUrl.txt");
            html.setOrignUrl(url);
            http.simpleGet(html);

            DocumentFragment listNode = DomTree.getNode(html.getContent(),html.getEncode());
            String nextPageUrl = null;
            if(listNode != null){
                //获取下一页的url
                NodeList nextPageNodeList = DomTree.commonList(CnkiOverseasPatentXpath.NEXT_PAGE_URL,listNode);
                if(nextPageNodeList != null && nextPageNodeList.getLength() > 0){
                    String nextPageName = nextPageNodeList.item(0).getTextContent();
                    if("下一页".equals(nextPageName)){
                        restartNum = 0;
                        nextPageUrl = "http://nvsm.cnki.net" + "/KNS/brief/brief.aspx" + nextPageNodeList.item(0).getAttributes().getNamedItem("href").getTextContent();
                        FileOperation.appendWrite("下一页的url为：" + nextPageUrl + "\n","config/cnkiPatentUrl.txt");
                    }
                }
            }
            if(nextPageUrl == null){
                if(restartNum == 0){
                    restartNum++;
                    logger.info("数据获取失败，15s后重新获取该页数据");
                    Thread.sleep(15 * 1000);

                    html = new HtmlInfo();
                    html.setEncode("UTF-8");
                    html.setUa(ua);
                    html.setOrignUrl(modelUrl);
                    String[]params = paramsModel.clone();
                    params[14] = paramsModel[14].replace("<searchKeyword>",searchKeyword);
                    http.simplePost(html,new ArrayList<>(Arrays.asList(params)));
                    url = restartUrl;
                    continue;
                }else if(restartNum == 1){
                    //防止restartNum仍然等于1，继续循环
                    restartNum++;
                }
            }

            //获取列表页每个专利的url,然后获取每个专利的信息
            NodeList listNodeList = DomTree.commonList(CnkiOverseasPatentXpath.PATENT_URL,listNode);

            List<CnkiPatentData> list = new ArrayList<>();
            //如果出现空指针异常，重新获取专利信息。如果重复一次仍然不行，则重新获取该页信息。
            int retryGetPatentInfo = 0;

            if(listNodeList != null){
                for(int i = 0;i < listNodeList.getLength();i++){
                    try{
                        String patentUrl = listNodeList.item(i).getTextContent();

                        CnkiPatentData patent = new CnkiPatentData();

                        logger.info("15s后获取第" + (i + 1) + "条专利的详细信息");
                        Thread.sleep(15 * 1000);
                        //"http://kns.cnki.net/kns/brief/brief.aspx?keyValue="+ lines[i] +"&S=1&pagename="
                        html.setReferUrl(URLEncoder.encode(url));
                        html.setOrignUrl(cnkiUrlPrefix + patentUrl);
                        //获取专利的url
                        patent.setPatentUrl(cnkiUrlPrefix + patentUrl);
                        patent.setSearchKeyword(searchKeyword);
                        try{
                            http.simpleGet(html);
                        }catch (NotFoundException e){
                            e.printStackTrace();
                            continue ;
                        }

                        DocumentFragment patentNode = DomTree.getNode(html.getContent(),html.getEncode());
                        //获取专利名称
                        NodeList patentNameNodeList = DomTree.commonList(CnkiOverseasPatentXpath.zhuanlimingcheng,patentNode);
                        if(patentNameNodeList != null && patentNameNodeList.getLength() > 0){
                            patent.setPatentName(patentNameNodeList.item(0).getTextContent());
                            logger.info("专利名称为：" + patent.getPatentName());
                        }
                        //获取申请号
                        NodeList requestNumNodeList = DomTree.commonList(CnkiOverseasPatentXpath.shenqinghao,patentNode);
                        if(requestNumNodeList != null && requestNumNodeList.getLength() > 0){
                            patent.setRequestNum(requestNumNodeList.item(0).getTextContent());
                        }
                        //获取申请日期
                        NodeList requestDateNodeList = DomTree.commonList(CnkiOverseasPatentXpath.shenqingri,patentNode);
                        if(requestDateNodeList != null && requestDateNodeList.getLength() > 0){
                            patent.setRequestDate(requestDateNodeList.item(0).getTextContent());
                        }
                        //获取公开号
                        NodeList publicNumNodeList = DomTree.commonList(CnkiOverseasPatentXpath.gongkaihao,patentNode);
                        if(publicNumNodeList != null && publicNumNodeList.getLength() > 0){
                            patent.setPublicNum(publicNumNodeList.item(0).getTextContent());
                            if(allPublicNum.contains(patent.getPublicNum())){
                                logger.info("该专利已保存到数据库，跳过采集");
                                continue;
                            }
                            allPublicNum = allPublicNum + patent.getPublicNum() + ",";
                        }
                        //获取公开日期
                        NodeList publicDateNodeList = DomTree.commonList(CnkiOverseasPatentXpath.gongkairi,patentNode);
                        if(publicDateNodeList != null && publicDateNodeList.getLength() > 0){
                            patent.setPublicDate(publicDateNodeList.item(0).getTextContent());
                        }
                        //获取申请人
                        NodeList proposerNodeList = DomTree.commonList(CnkiOverseasPatentXpath.shenqingren,patentNode);
                        if(proposerNodeList != null && proposerNodeList.getLength() > 0){
                            patent.setProposer(proposerNodeList.item(0).getTextContent());
                        }
                        //获取发明人
                        NodeList originatorNodeList = DomTree.commonList(CnkiOverseasPatentXpath.famingren,patentNode);
                        if(originatorNodeList != null && originatorNodeList.getLength() > 0){
                            patent.setOriginator(originatorNodeList.item(0).getTextContent());
                        }
                        //获取摘要
                        NodeList briefNodeList = DomTree.commonList(CnkiOverseasPatentXpath.zhaiyao,patentNode);
                        if(briefNodeList != null && briefNodeList.getLength() > 0){
                            patent.setBrief(briefNodeList.item(0).getTextContent());
                        }
                        //获取主分类号
                        NodeList mainClassificationNumNodeList = DomTree.commonList(CnkiOverseasPatentXpath.fenleihao,patentNode);
                        if(mainClassificationNumNodeList != null && mainClassificationNumNodeList.getLength() > 0){
                            patent.setMainClassificationNum(mainClassificationNumNodeList.item(0).getTextContent());
                        }
                        list.add(patent);
                        retryGetPatentInfo = 0;
                    }catch (NullPointerException e){
                        e.printStackTrace();
                        if(retryGetPatentInfo <= 1){
                            logger.info("信息采集空指针异常，重新获取该专利信息");
                            i--;
                            retryGetPatentInfo++;
                        }
                    }
                }
                //去除垃圾数据
                for(int a = list.size() - 1;a>=0;a--){
                    CnkiPatentData patent =list.get(a);
                    if(patent.getPatentName() == null || "".equals(patent.getPatentName())){
                        list.remove(a);
                    }
                }
                logger.info("该页专利的详细数据获取完成，保存到数据库");
                CnkiOverseasPatentSaveDataToSql.saveCnkiPaperToSql(list);
                logger.info("数据保存完成，15s后获取下一页的数据");
                Thread.sleep(15 * 1000);
            }
            url = nextPageUrl;
        }
    }

    public static void queryNewPatent(String url) throws InterruptedException {

        //重新启动次数   判断依据为：如果下一页的url获取失败，那么就重新获取该页的数据，重新获取一次
        int restartNum = 0;

        restart:while (url != null || restartNum <= 1){
            //记录下当前url，当程序出现问题时重新请求该url
            String restartUrl = url;
            CnkiPatentConfig.entranceUrl = url;
            logger.info("获取关键词《" + searchKeyword + "》的最新专利信息。当前列表URL为：" + url);
            html.setOrignUrl(url);
            http.simpleGet(html);

            DocumentFragment listNode = DomTree.getNode(html.getContent(),html.getEncode());
            String nextPageUrl = null;
            if(listNode != null){
                //获取下一页的url
                NodeList nextPageNodeList = DomTree.commonList(CnkiOverseasPatentXpath.NEXT_PAGE_URL,listNode);
                if(nextPageNodeList != null && nextPageNodeList.getLength() > 0){
                    String nextPageName = nextPageNodeList.item(0).getTextContent();
                    if("下一页".equals(nextPageName)){
                        restartNum = 0;
                        nextPageUrl = "http://nvsm.cnki.net" + "/KNS/brief/brief.aspx" + nextPageNodeList.item(0).getAttributes().getNamedItem("href").getTextContent();
                    }
                }
            }
            if(nextPageUrl == null){
                if(restartNum == 0){
                    restartNum++;
                    logger.info("数据获取失败，15s后重新获取该页数据");
                    Thread.sleep(15 * 1000);

                    html = new HtmlInfo();
                    html.setEncode("UTF-8");
                    html.setUa(ua);
                    html.setOrignUrl(modelUrl);
                    String[]params = paramsModel.clone();
                    params[14] = paramsModel[14].replace("<searchKeyword>",searchKeyword);
                    http.simplePost(html,new ArrayList<>(Arrays.asList(params)));
                    url = restartUrl;
                    continue;
                }else if(restartNum == 1){
                    //防止restartNum仍然等于1，继续循环
                    restartNum++;
                }
            }

            //获取列表页每个专利的url,然后获取每个专利的信息
            NodeList listNodeList = DomTree.commonList(CnkiOverseasPatentXpath.PATENT_URL,listNode);

            List<CnkiPatentData> list = new ArrayList<>();
            //如果出现空指针异常，重新获取专利信息。如果重复一次仍然不行，则重新获取该页信息。
            int retryGetPatentInfo = 0;

            if(listNodeList != null){
                for(int i = 0;i < listNodeList.getLength();i++){
                    try{
                        String patentUrl = listNodeList.item(i).getTextContent();

                        CnkiPatentData patent = new CnkiPatentData();

                        logger.info("15s后获取第" + (i + 1) + "条专利的详细信息");
                        Thread.sleep(15 * 1000);
                        //"http://kns.cnki.net/kns/brief/brief.aspx?keyValue="+ lines[i] +"&S=1&pagename="
                        html.setReferUrl(URLEncoder.encode(url));
                        html.setOrignUrl(cnkiUrlPrefix + patentUrl);
                        //获取专利的url
                        patent.setPatentUrl(cnkiUrlPrefix + patentUrl);
                        patent.setSearchKeyword(searchKeyword);
                        try{
                            http.simpleGet(html);
                        }catch (NotFoundException e){
                            e.printStackTrace();
                            continue ;
                        }

                        DocumentFragment patentNode = DomTree.getNode(html.getContent(),html.getEncode());
                        //获取专利名称
                        NodeList patentNameNodeList = DomTree.commonList(CnkiOverseasPatentXpath.zhuanlimingcheng,patentNode);
                        if(patentNameNodeList != null && patentNameNodeList.getLength() > 0){
                            patent.setPatentName(patentNameNodeList.item(0).getTextContent());
                            logger.info("专利名称为：" + patent.getPatentName());
                        }
                        //获取申请号
                        NodeList requestNumNodeList = DomTree.commonList(CnkiOverseasPatentXpath.shenqinghao,patentNode);
                        if(requestNumNodeList != null && requestNumNodeList.getLength() > 0){
                            patent.setRequestNum(requestNumNodeList.item(0).getTextContent());
                        }
                        //获取申请日期
                        NodeList requestDateNodeList = DomTree.commonList(CnkiOverseasPatentXpath.shenqingri,patentNode);
                        if(requestDateNodeList != null && requestDateNodeList.getLength() > 0){
                            patent.setRequestDate(requestDateNodeList.item(0).getTextContent());
                        }
                        //获取公开号
                        NodeList publicNumNodeList = DomTree.commonList(CnkiOverseasPatentXpath.gongkaihao,patentNode);
                        if(publicNumNodeList != null && publicNumNodeList.getLength() > 0){
                            patent.setPublicNum(publicNumNodeList.item(0).getTextContent());
                            if(allPublicNum.contains(patent.getPublicNum())){
                                logger.info("该专利已保存到数据库，跳过采集");
                                continue;
                            }
                            allPublicNum = allPublicNum + patent.getPublicNum() + ",";
                        }
                        //获取公开日期
                        NodeList publicDateNodeList = DomTree.commonList(CnkiOverseasPatentXpath.gongkairi,patentNode);
                        if(publicDateNodeList != null && publicDateNodeList.getLength() > 0){
                            patent.setPublicDate(publicDateNodeList.item(0).getTextContent());
                        }
                        //获取申请人
                        NodeList proposerNodeList = DomTree.commonList(CnkiOverseasPatentXpath.shenqingren,patentNode);
                        if(proposerNodeList != null && proposerNodeList.getLength() > 0){
                            patent.setProposer(proposerNodeList.item(0).getTextContent());
                        }
                        //获取发明人
                        NodeList originatorNodeList = DomTree.commonList(CnkiOverseasPatentXpath.famingren,patentNode);
                        if(originatorNodeList != null && originatorNodeList.getLength() > 0){
                            patent.setOriginator(originatorNodeList.item(0).getTextContent());
                        }
                        //获取摘要
                        NodeList briefNodeList = DomTree.commonList(CnkiOverseasPatentXpath.zhaiyao,patentNode);
                        if(briefNodeList != null && briefNodeList.getLength() > 0){
                            patent.setBrief(briefNodeList.item(0).getTextContent());
                        }
                        //获取主分类号
                        NodeList mainClassificationNumNodeList = DomTree.commonList(CnkiOverseasPatentXpath.fenleihao,patentNode);
                        if(mainClassificationNumNodeList != null && mainClassificationNumNodeList.getLength() > 0){
                            patent.setMainClassificationNum(mainClassificationNumNodeList.item(0).getTextContent());
                        }
                        list.add(patent);
                        retryGetPatentInfo = 0;
                    }catch (NullPointerException e){
                        e.printStackTrace();
                        if(retryGetPatentInfo <= 1){
                            logger.info("信息采集空指针异常，重新获取该专利信息");
                            i--;
                            retryGetPatentInfo++;
                        }
                    }
                }
                if(list.size() == 0){
                    logger.info("关键词《" + searchKeyword + "》无新数据");
                    return;
                }
                //去除垃圾数据
                for(int a = list.size() - 1;a>=0;a--){
                    CnkiPatentData patent =list.get(a);
                    if(patent.getPatentName() == null || "".equals(patent.getPatentName())){
                        list.remove(a);
                    }
                }
                logger.info("该页专利的详细数据获取完成，保存到数据库");
                CnkiOverseasPatentSaveDataToSql.saveCnkiPaperToSql(list);
                logger.info("数据保存完成，15s后获取下一页的数据");
                Thread.sleep(15 * 1000);
            }
            url = nextPageUrl;
        }
    }
}
